{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "50859518",
   "metadata": {},
   "source": [
    "# lab 2\n",
    "\n",
    "## explanaions\n",
    "\n",
    "- (1)、(2)均在原数据基础上处理\n",
    "- (3)输出一条即可\n",
    "- (4)用总帧数除以时间\n",
    "- (5)尽可能均匀选取\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b5a0d2e",
   "metadata": {},
   "source": [
    "## Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "326b2fbc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# pre-processing\n",
    "f = open('gaze.csv') # read original data\n",
    "\n",
    "columnNames = f.readline().strip().split(',') # get column names\n",
    "\n",
    "# print(columnNames) # print column names\n",
    "\n",
    "from collections import namedtuple # use namedtuple to process data pieces\n",
    "\n",
    "DataPiece = namedtuple('DataPiece', ' '.join(columnNames)) # set namedtuple\n",
    "\n",
    "# print(DataPiece._fields) # get field names\n",
    "\n",
    "datas = [] # original data in namedtuple format\n",
    "\n",
    "for line in f.readlines():\n",
    "    datas.append(DataPiece._make(line.strip().split(','))) # namedtuple._make(), transform format from str to namedtuple\n",
    "\n",
    "# print(datas[:10]) # debug"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "695c7846",
   "metadata": {},
   "source": [
    "## exercise 1:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b65e69da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "87442\n"
     ]
    }
   ],
   "source": [
    "# exercise 1\n",
    "# (1)confidence为信度,confidence越高代表本条数据越可信,\n",
    "# 只保留0.9(含0.9)及以上confidence的数据,其他行数据丢弃-输出剩余数据条数。\n",
    "\n",
    "data1 = list(filter(lambda x: float(x.confidence) >= 0.9, datas)) # use filter to process data\n",
    "\n",
    "# print(ans1[:10]) # debug\n",
    "\n",
    "print(len(data1)) # get the answer to exercise 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ea7f26b",
   "metadata": {},
   "source": [
    "## exercise 2:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "23145294",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120036\n"
     ]
    }
   ],
   "source": [
    "# exercise 2\n",
    "# (2）列norm_pos x和normpos x为预测的归一化后的坐标值，,\n",
    "# 假设norm pos x和norm pos y符合正态分布,利用3 sigma原则去除无效值,\n",
    "# 即去掉>mean+3sigmaLX区<mean-3sigma的值：输出剩余数据条数.\n",
    "\n",
    "import math\n",
    "\n",
    "totalOfDatas = len(datas)\n",
    "# print('totalOfDatas:', totalOfDatas) # debug\n",
    "\n",
    "\n",
    "# get mean of X and Y\n",
    "sumX, sumY = 0.0, 0.0\n",
    "for piece in datas:\n",
    "    sumX += float(piece.norm_pos_x)\n",
    "    sumY += float(piece.norm_pos_y)\n",
    "meanX = sumX / totalOfDatas\n",
    "meanY = sumY / totalOfDatas\n",
    "# print('meanX:', meanX, 'meanY:', meanY) # debug\n",
    "\n",
    "\n",
    "# get variance of X and Y\n",
    "varianceX, varianceY = 0.0, 0.0\n",
    "for piece in datas:\n",
    "    varianceX += pow(float(piece.norm_pos_x) - meanX, 2)\n",
    "    varianceY += pow(float(piece.norm_pos_y) - meanY, 2)\n",
    "varianceX /= totalOfDatas\n",
    "varianceY /= totalOfDatas\n",
    "sigmaX =  math.sqrt(varianceX)\n",
    "sigmaY =  math.sqrt(varianceY)\n",
    "\n",
    "# print('varianceX:', varianceX, 'varianceY:', varianceY) # debug\n",
    "# print('sigmaX:', sigmaX, 'sigmaY:', sigmaY) # debug\n",
    "# print(f'x in:[{meanX - 3 * sigmaX}, {meanX + 3 * sigmaX}]')\n",
    "# print(f'y in:[{meanY - 3 * sigmaY}, {meanY + 3 * sigmaY}]')\n",
    "\n",
    "data2 = list(filter(lambda p: meanX - 3 * sigmaX < float(p.norm_pos_x) < meanX + 3 * sigmaX and meanY - 3 * sigmaY < float(p.norm_pos_y) < meanY + 3 * sigmaY, datas))\n",
    "# print(data2[:10]) # debug\n",
    "\n",
    "print(len(data2)) # get the answer to exercise 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce57799a",
   "metadata": {},
   "source": [
    "## exercise 3:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2378c48c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1970-01-02 00:29:10.776780+00:00\n"
     ]
    }
   ],
   "source": [
    "# (3） gaze timestamp列是时间戳(单位为s),为了方便分析人员查看,将其改为\"1970-01-02T17:44:30.225707+0000\"的形式\n",
    "\n",
    "from datetime import datetime, timezone # to transform the format of time\n",
    "\n",
    "for idx, piece in enumerate(datas):\n",
    "    datas[idx] = datas[idx]._replace(gaze_timestamp = datetime.fromtimestamp(float(piece.gaze_timestamp), timezone.utc)) # namedtuple = namedtuple._replace, replace the timestamp in ISO format.\n",
    "    # datas[idx] = datas[idx]._replace(gaze_timestamp = datetime.fromtimestamp(float(piece.gaze_timestamp), timezone.utc).isoformat()) # namedtuple = namedtuple._replace, replace the timestamp in ISO format.\n",
    "\n",
    "print(datas[0].gaze_timestamp) # get the answer to exercise 3, str(datetime) is equal to datetime.isoformat() as default."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5679412",
   "metadata": {},
   "source": [
    "## exercise 4:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b55128bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "427.6261076608263\n"
     ]
    }
   ],
   "source": [
    "# (4）高帧率采集设备在采集每帧时有一定的偏差,请问该数据的采样率是多少？\n",
    "\n",
    "from datetime import timedelta # to calculate the seconds of time delta\n",
    "\n",
    "startTime = datas[0].gaze_timestamp\n",
    "endTime = datas[-1].gaze_timestamp\n",
    "# print(startTime.isoformat(), endTime.isoformat()) # debug\n",
    "totalTimeDelta = endTime - startTime\n",
    "totalTimeDuration = timedelta.total_seconds(totalTimeDelta) # transform format\n",
    "# print(totalOfDatas, totalTimeDuration) # debug\n",
    "\n",
    "print(totalOfDatas / totalTimeDuration) # get the answer to exercise 4"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6aba0aae",
   "metadata": {},
   "source": [
    "## exercise 5:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3a18a2d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1970-01-02 00:29:10.779759+00:00\n",
      "1970-01-02 00:29:10.789480+00:00\n",
      "1970-01-02 00:29:10.801140+00:00\n",
      "1970-01-02 00:29:10.809749+00:00\n",
      "1970-01-02 00:29:10.821819+00:00\n",
      "1970-01-02 00:29:10.829663+00:00\n",
      "1970-01-02 00:29:10.839575+00:00\n",
      "1970-01-02 00:29:10.849505+00:00\n",
      "1970-01-02 00:29:10.859561+00:00\n",
      "1970-01-02 00:29:10.869620+00:00\n",
      "1970-01-02 00:29:10.879584+00:00\n",
      "1970-01-02 00:29:10.889996+00:00\n",
      "1970-01-02 00:29:10.899511+00:00\n",
      "1970-01-02 00:29:10.909398+00:00\n",
      "1970-01-02 00:29:10.919423+00:00\n",
      "1970-01-02 00:29:10.931877+00:00\n",
      "1970-01-02 00:29:10.940892+00:00\n",
      "1970-01-02 00:29:10.951503+00:00\n",
      "1970-01-02 00:29:10.959502+00:00\n",
      "1970-01-02 00:29:10.969507+00:00\n"
     ]
    }
   ],
   "source": [
    "# (5）高帧率采集设备在采集每帧的时需要打上时间戳,但是时间戳有误差,每一帧时间戳不是严格按照采样\n",
    "# 率,为了后期时序数据处理方便，将gaze_timestamp列按照100Hz的帧率重新采样,输出处理后的数据集的\n",
    "# 前20行数据，时间戳格式不限。\n",
    "\n",
    "\n",
    "timeSeries = [] # resampling time series\n",
    "tmp = round(startTime.timestamp(), 2) # temporary variable for resampling time\n",
    "while tmp < endTime.timestamp():\n",
    "    timeSeries.append(datetime.fromtimestamp(tmp, timezone.utc))\n",
    "    tmp = round(tmp + 0.01, 2) # 100 Hz\n",
    "# print(timeSeries[:20]) # debug\n",
    "data4 = []\n",
    "i = 0 # iterator variable for datas\n",
    "for stime in timeSeries:\n",
    "    while i < totalOfDatas and abs(datas[i+1].gaze_timestamp - stime) < abs(datas[i].gaze_timestamp - stime): # find the closest timestamp to `stime`\n",
    "        i += 1\n",
    "    data4.append(datas[i])\n",
    "\n",
    "print('\\n'.join([str(piece.gaze_timestamp) for piece in data4[:20]]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
